Sys.setlocale(category = 'LC_ALL', locale = 'English_United States.1252'); 
.libPaths(file.path(R.home(),'library')); 
setwd('/Users/liukangning/Desktop/EmpowerROS/dataMerge/BODY'); 
pkgs<-c('haven','readxl','foreign','plyr'); 
for (g in pkgs) {if (!(g %in% rownames(installed.packages()))) install.packages(g,repos='https://cloud.r-project.org');} 
library(haven); 
library(readxl); 
library(foreign); 
library(plyr); 
idvname<- c(''); 
dfname <- c(''); 
dvars<- list(c('')); 
idset<- c(); idsetX<-idset; idsetX[idset==2]<-0; idsetX1<-paste(idsetX,collapse=''); 
readdataFile<-function(inFile, fmt="", skips=0, nrows=-1, setCols=NA) {
  if (fmt == "") {
    fmt<-toupper(rev(unlist(strsplit(inFile, "\\.")))[1])
    if (fmt == "XLS") {
      dtxt<-read.table(inFile, header=TRUE, quote="\"'", as.is=TRUE, na.string=c("NA",".",""," "), sep="", nrows=3, skip=0)
      if (!(nrow(dtxt) == 0)) {fmt = ""; rm(dtxt); gc();}
    } else {
      if (fmt == "CSV") fmt = "";
      if (fmt == "TXT") fmt = "";
    }
  }
  if (fmt == "") {
    dtxt<-read.table(inFile, header=TRUE, quote="\"'", as.is=TRUE, na.string=c("NA",".",""," "), sep="", nrows=3, skip=0)
    dcsv<-read.table(inFile, header=TRUE, quote="\"'", as.is=TRUE, na.string=c("NA",".",""," "), sep=",", nrows=3, skip=0)
    dcs2<-read.table(inFile, header=TRUE, quote="\"'", as.is=TRUE, na.string=c("NA",".",""," "), sep=";", nrows=3, skip=0)
    n3<-c(ncol(dtxt), ncol(dcsv), ncol(dcs2))
    fmt<- c("TXT","CSV","CSV2")[which(n3==max(n3))]
    rm(dtxt, dcsv, dcs2); gc();  
  } 
  if (fmt=="CSV") {
    d<-read.table(inFile, header=TRUE, quote="\"'", as.is=TRUE, na.string=c("NA",".",""," "), comment.char="",
    sep=",", nrows=nrows, skip=skips, blank.lines.skip=TRUE, fill=TRUE, colClasses=setCols)
  }
  if (fmt=="CSV2") {
    d<-read.table(inFile, header=TRUE, quote="\"'", as.is=TRUE, na.string=c("NA",".",""," "), comment.char="",
    sep=";", nrows=nrows, skip=skips, blank.lines.skip=TRUE, fill=TRUE, colClasses=setCols)
  }  
  if (fmt=="TXT") {
    d<-read.table(inFile, header=TRUE, quote="\"'", as.is=TRUE, na.string=c("NA",".",""," "), comment.char="",
    sep="", nrows=nrows, skip=skips, blank.lines.skip=TRUE, fill=TRUE, colClasses=setCols)
  }
  if (fmt=="TAB") {
    d<-read.table(inFile, header=TRUE, quote="\"'", as.is=TRUE, na.string=c("NA",".",""," "), comment.char="",
    sep="\t", nrows=nrows, skip=skips, blank.lines.skip=TRUE, fill=TRUE, colClasses=setCols)
  }
  if (fmt=="RDATA") d<-get(load(inFile));
  if (fmt=="SAS7BDAT") d<- data.frame(read_sas(inFile));
  if (fmt=="XPT") d<-data.frame(read_xpt(inFile));
  if (fmt=="SAV") d<-read.spss(inFile,to.data.frame = TRUE,use.value.labels = FALSE);
  if (fmt=="DTA") d<-data.frame(read_dta(inFile));
  if (fmt=="REC") d<-read.epiinfo(inFile);
  if (fmt=="XLSX" || fmt == "XLS") d<-data.frame(read_excel(inFile));

  if (file.info(inFile)$size < 300*1024*1024) {  
    d<-d[(apply(is.na(d),1,sum)<ncol(d)),]
    d<-d[,(apply(is.na(d),2,sum)<nrow(d))]
  }
  cname<-gsub("\\.+","\\.",colnames(d))
  cname<-gsub("[,;|]","",cname)
  cx <- endsWith(cname,".")
  while (sum(cx)>0) {cname[cx] <- substr(cname[cx],1,nchar(cname[cx])-1); cx <- endsWith(cname,".");}
  cname <- toupper(cname)  
  cnamef <- table(cname); rep <- names(cnamef[cnamef > 1]); 
  if (length(rep)>0) {
    for (i in 1:length(rep)) cname[which(cname==rep[i])]<-paste(rep[i], 1:cnamef[rep[i]], sep=".")
  }
  rm(cnamef, rep); gc();
  colnames(d)<-cname  
  return(d)
}
 
dfout<-NULL; nd<-length(dfname); 
oo<-NULL; 
dname<-list.files(path='.', pattern='.(csv|txt|xls|xlsx|rec|sas7bdat|xpt|sav|dta|rdata)', all.files=FALSE, full.names=FALSE, ignore.case=TRUE); 
for (fname in dname) {dti<-readdataFile(fname); oo<-rbind(oo,c(fname,nrow(dti),paste(colnames(dti),collapse=','))); rm(dti); gc();} 
write.table(oo, file='_tmp.lst', sep='	', col.names=FALSE, row.names=FALSE, append=FALSE, quote=FALSE);